In [1]:
%matplotlib inline
import warnings
import seaborn as sns
warnings.filterwarnings('ignore')
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.decomposition import PCA
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score,confusion_matrix, classification_report,roc_auc_score
from scipy.stats import zscore
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from scipy.stats import iqr
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
In [2]:
data = pd.read_csv("vehicle.csv")
print('\n\nshape of data:', data.shape)
print('\n\nCount of target variable:\n\n', data['class'].value_counts(),'\n\nData types of attributes:\n\n')
print(data.info())
print('\n\n First five rows of data:\n\n', data.head())
print('\n\n Few statistical values of attributes:\n\n', data.describe())
In the above block of code¶
- We read the data set and try to eye ball the data¶
- There are 846 rows and 19 columns(attributes)¶
- We check the target variable and see that there is 429 instances of car, 218 instnaces of bus and 199 instances of van¶
- We check the data types of all attributes and see that compactness, max.length_aspect_ratio, max.length_rectangularity and hollows_ratio are integer types, class is type object and rest all attributes are float types¶
- We check head of the data set and see 5 rows just to get the feel¶
- We also see the statistical values of the attributes and see the data are mostly non decimal and variant from each other only in tens and hundreds place.¶
In [5]:
En = LabelEncoder()
columns = data.columns
data['class'] = En.fit_transform(data['class'])
print(data['class'].head())
- In this above block of code we use one hot encoding to convert our categorical variable(class) for convenience.¶
In [6]:
newdata = data.copy()
print(pd.DataFrame(newdata.isnull().sum(), columns= ['Number of missing values']))
- Here we check missing values and we see a few missing values here and there in some attributes.¶
In [7]:
X = newdata.iloc[:,0:19]
imputer = SimpleImputer(missing_values=np.nan, strategy='median', verbose=1)
transformed_values = imputer.fit_transform(X)
column = X.columns
newdata = pd.DataFrame(transformed_values, columns = column)
print(pd.DataFrame(newdata.isnull().sum(), columns= ['Number of missing values']))
- Here we replace the missing values by the median of the attribute.¶
In [8]:
plt.figure(figsize= (20,15))
plt.subplot(9,3,1)
sns.boxplot(x= newdata['pr.axis_aspect_ratio'])
plt.subplot(9,3,2)
sns.boxplot(x= newdata.skewness_about)
plt.subplot(9,3,3)
sns.boxplot(x= newdata.scaled_variance)
plt.figure(figsize= (20,15))
plt.subplot(9,3,4)
sns.boxplot(x= newdata['radius_ratio'])
plt.subplot(9,3,5)
sns.boxplot(x= newdata['scaled_radius_of_gyration.1'])
plt.subplot(9,3,6)
sns.boxplot(x= newdata['scaled_variance.1'])
plt.figure(figsize= (20,15))
plt.subplot(9,3,7)
sns.boxplot(x= newdata['max.length_aspect_ratio'])
plt.subplot(9,3,8)
sns.boxplot(x= newdata['skewness_about.1'])
plt.show()
Q1 = newdata.quantile(0.25)
Q3 = newdata.quantile(0.75)
IQR = Q3 - Q1
newdata2 = newdata[~((newdata < (Q1 - 1.5 * IQR)) |(newdata > (Q3 + 1.5 * IQR))).any(axis=1)]
print('\n\n After removal of outliers:\n\n')
plt.figure(figsize= (20,15))
plt.subplot(9,3,1)
sns.boxplot(x= newdata2['pr.axis_aspect_ratio'])
plt.subplot(9,3,2)
sns.boxplot(x= newdata2.skewness_about)
plt.subplot(9,3,3)
sns.boxplot(x= newdata2.scaled_variance)
plt.figure(figsize= (20,15))
plt.subplot(9,3,4)
sns.boxplot(x= newdata2['radius_ratio'])
plt.subplot(9,3,5)
sns.boxplot(x= newdata2['scaled_radius_of_gyration.1'])
plt.subplot(9,3,6)
sns.boxplot(x= newdata2['scaled_variance.1'])
plt.figure(figsize= (20,15))
plt.subplot(9,3,7)
sns.boxplot(x= newdata2['max.length_aspect_ratio'])
plt.subplot(9,3,8)
sns.boxplot(x= newdata2['skewness_about.1'])
plt.show()
In [9]:
newdata2.hist(bins=20, figsize=(60,40))
plt.show()
In [10]:
skewness = newdata2.skew()
print("skewValue of dataframe attributes: \n", skewness)
In [12]:
f, ax = plt.subplots(1, 5, figsize=(30,5))
sns.distplot(newdata2["scaled_variance.1"],bins=10, ax= ax[0])
sns.distplot(newdata2["scaled_variance"],bins=10, ax=ax[1])
sns.distplot(newdata2["skewness_about.1"],bins=10, ax= ax[2])
sns.distplot(newdata2["skewness_about"],bins=10, ax=ax[3])
sns.distplot(newdata2["scatter_ratio"],bins=10, ax=ax[4])
plt.show()
- Here we take a closer look at our skewed attributes.¶
In [13]:
def correlation_heatmap(dataframe,l,w):
correlation = dataframe.corr()
plt.figure(figsize=(l,w))
sns.heatmap(correlation, vmax=1, square=True,annot=True,cmap='viridis')
plt.show();
corr= newdata2.drop('class', axis=1)
correlation_heatmap(corr, 30,15)
- here we check correlation between variables using a heatmap(Pearson's coeff)¶
These attributes have Strong correlation ~ 0.8 and above¶
- Scaled Variance & Scaled Variance.1¶
- skewness_about_2 and hollow_ratio¶
- ditance_circularity and radius_ratio¶
- scaled _variance and scaled_radius_of_gyration¶
- pr.axis_recatngularity and max.length_recatngularity¶
- scatter_ratio and elongatedness¶
- elongatedness and pr.axis_rectangularity¶
These attributes have weak or no correlation¶
- max_length_aspect_ratio & radius_ratio¶
- pr.axis_aspect_ratio & max_length_aspect_ratio¶
- scaled_radius_gyration & scaled_radisu_gyration.1¶
- scaled_radius_gyration.1 & skewness_about¶
- skewness_about & skewness_about.1¶
- skewness_about.1 and skewness_about.2¶
In [15]:
sns.pairplot(newdata2, diag_kind="kde")
Out[15]:
In th above pair plot analysis and adding up to our correlation analysis we see a lot of attrinutes which are highly correlated. So we can decide to remove columns which are highly correlated.There are 8 such columns which can be considered not important:¶
- max.length_rectangularity¶
- scaled_radius_of_gyration¶
- skewness_about.2¶
- scatter_ratio¶
- elongatedness¶
- pr.axis_rectangularity¶
- scaled_variance¶
- scaled_variance.1¶
The rest columns can be considered important, but we will eventually get those using PCA¶
In [16]:
X = newdata2.iloc[:,0:18].values
y = newdata2.iloc[:,18].values
S = StandardScaler()
X_scaled = S.fit_transform(X)
In the above block of code we normalise/scale our data set¶
In [18]:
O_X_train,O_X_test,O_y_train,O_y_test = train_test_split(X_scaled,y,test_size=0.30,random_state=1)
print("{0:0.2f}% data is in training set".format((len(O_X_train)/len(data.index)) * 100))
print("{0:0.2f}% data is in test set".format((len(O_X_test)/len(data.index)) * 100))
In the above block of code we split the data into train:test(70%:30%)¶
In [19]:
svc = SVC()
svc.fit(O_X_train,O_y_train)
O_y_predict = svc.predict(O_X_test)
print("Model Score On Original Data ",svc.score(O_X_test, O_y_test)*100, '%')
print('\n\nConfusion matrix of Original Data:\n', confusion_matrix(O_y_test, O_y_predict ))
In [21]:
kfold = KFold(n_splits=10, random_state=1)
results1 = cross_val_score(svc, X_scaled, y, cv=kfold)
print(results1)
print("Accuracy: %.3f%% (%.3f%%)" % (results1.mean()*100.0, results1.std()*100.0))
- Since a single test set can be misleading, we also perform a k fold cross validation getting an accuracy of 96.26% with a buffer of 1.9 %¶
In [22]:
cov_matrix = np.cov(X_scaled.T)
print("\n\nCovariance_matrix shape:",cov_matrix.shape)
print("\n\nCovariance_matrix:\n\n",cov_matrix)
eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)
print('\n\nEigen Vectors:\n\n', eigenvectors)
print('\n Eigen Values: \n\n', eigenvalues)
- In the above block of code we calculate the covariance matrix, eigen values and eigen vectors required for PCA¶
In [23]:
eig_pairs = [(eigenvalues[index], eigenvectors[:,index]) for index in range(len(eigenvalues))]
eig_pairs.sort()
eig_pairs.reverse()
print(eig_pairs)
eigvalues_sorted = [eig_pairs[index][0] for index in range(len(eigenvalues))]
eigvectors_sorted = [eig_pairs[index][1] for index in range(len(eigenvalues))]
print('\nEigenvalues in descending order:\n\n', eigvalues_sorted)
In [24]:
summation = sum(eigenvalues)
variance = [(i/summation) for i in sorted(eigenvalues, reverse=True)] # an array of variance explained by each
# eigen vector... there will be 18 entries as there are 18 eigen vectors)
cum_variance=np.cumsum(variance) # an array of cumulative variance. There will be 18 entries with 18 th entry
# cumulative reaching almost 100%
plt.bar(range(1,19), variance, alpha=0.5, align='center', label='individual explained variance')
plt.step(range(1,19),cum_variance, where= 'mid', label='cumulative explained variance')
plt.ylabel('Explained variance ratio')
plt.xlabel('Principal components')
plt.legend(loc = 'best')
plt.show()
In [25]:
P_reduce = np.array(eigvectors_sorted[0:8])
X_std_8D = np.dot(X_scaled,P_reduce.T)
reduced_pca = pd.DataFrame(X_std_8D)
reduced_pca
Out[25]:
- Here we get the Principle components as our main features¶
In [26]:
sns.pairplot(reduced_pca, diag_kind='kde')
Out[26]:
- As we can see in the pairplot there is not correlation between any of our attributes¶
In [27]:
pca_X_train,pca_X_test,pca_y_train,pca_y_test = train_test_split(reduced_pca,y,test_size=0.30,random_state=1)
print("{0:0.2f}% data is in training set".format((len(pca_X_train)/len(data.index)) * 100))
print("{0:0.2f}% data is in test set".format((len(pca_X_test)/len(data.index)) * 100))
Above Step 3 but with PCA reducded data (splitting into train:test)¶
In [28]:
svc1 = SVC()
svc1.fit(pca_X_train,pca_y_train)
pca_y_predict = svc1.predict(pca_X_test)
print("Model Score On reduced Data ",svc1.score(pca_X_test, pca_y_test)*100, '%')
print('\n\nConfusion matrix of Original Data:\n', confusion_matrix(pca_y_test, pca_y_predict ))
- Here we train a SVM model with reduced PCA data set and see that accuracy is 96.72 %. This is less than our normal model but not a lot of accuracy is compromised given that we removed 10 attributes¶
In [29]:
kfold = KFold(n_splits=10, random_state=1)
results2 = cross_val_score(svc1,reduced_pca, y, cv=kfold)
print(results2)
print("Accuracy: %.3f%% (%.3f%%)" % (results2.mean()*100.0, results2.std()*100.0))
- Above is k fold cross validation for PCA reduced data set¶
In [30]:
print("Classification Report For Raw Data:", "\n", classification_report(O_y_test,O_y_predict))
print("Classification Report For PCA:","\n", classification_report(pca_y_test,pca_y_predict))
print("\nCross Validation Accuracy of Raw data: %.3f%% (%.3f%%)" % (results1.mean()*100.0, results1.std()*100.0))
print("\nCross Validation Accuracy of PCA data: %.3f%% (%.3f%%)" % (results2.mean()*100.0, results2.std()*100.0))